import requests
from bs4 import BeautifulSoup

url = "https://www.meitulu.com/gangtai/28.html"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}


def crawler(url):
    source_code = requests.get(url, headers=headers)
    try:
        source_code.raise_for_status()
        apparent_encoding = source_code.apparent_encoding
        source_code.encoding = apparent_encoding
        # print(source_code.text)
        soup = BeautifulSoup(source_code.text, "lxml")
        print(soup.head)
        print(soup.head.contents)
    except requests.ConnectionError as e:
        print(url + " : Connection error !!!!")
        print(e)
    except requests.HTTPError as e:
        print(url + " : Connection error !!!!")
        print(e)


if __name__ == "__main__":
    crawler(url)
